# -*- coding: utf-8 -*-
import scrapy,requests
import scrapy.http
import urllib.request
from scrapy import Spider
import scrapy,re,logging,requests,json
from scrapy.selector import Selector
from scrapy.http import Request
from tianshan.items import TianshanItem

base_url = 'https://edu.hellobi.com/course/explore?page='
item = TianshanItem()
class TsSpider(scrapy.Spider):
    name = 'ts'
    allowed_domains = ['hellobi.com']
    # start_urls = ['https://edu.hellobi.com/course/explore']


    def start_requests(self):
        for i in range(1,16):
            url = base_url+str(i)
            yield Request(url =url,callback=self.parse )

    def parse(self, response):

        texts = response.xpath('//div[@class="caption"]/h3/a/@href').extract()
        for link in texts:

            yield Request(url=link,callback=self.parse_lesson)

    def parse_lesson(self,response):

        item['title']=response.xpath('//div[@class="container"]/ol[@class="breadcrumb"]/li[@class="active"]/text()').extract()
        item['Link'] = response.url
        item['price'] = response.xpath('//div[@class="course-price"]/span[@class="price-free"]/text()').extract()
        if not item['price']:
            item['price'] = response.xpath('//div[@class="course-price"]/span[@class="m-price"]/span[@class="price-expense"]/text()').extract()
        print(item['title'],item['Link'],item['price'])
        # this_url = response.url
        # data = urllib.request.urlopen(this_url).read().decode('utf-8')





